#!/usr/bin/env python  
#coding=utf-8

#导入所需包
import os
import sqlite3
from pymmseg import mmseg

#使用默认的字典
#mmseg.dict_load_defaults()


#需要过滤的字符
filterList = ['\xe3\x80\x80', ' ','（', '）', '(', ')', '•', '。', '，', '…', '·', '；', '、', \
              '？', '“', '”','·', '：', '！',  \
              '了', '的', '着','在', '和', '被', '饰']


#设定脚本所在目录为工作目录
os.chdir(os.path.dirname(__file__))

#使用自定义字典
mmseg.dict_load_words('customize_words.dic')
mmseg.dict_load_chars('customize_chars.dic')

#数据库连接
conn = sqlite3.connect('doubanMoviesDB.db')
curs = conn.cursor()

#抽取电影描述
sourceQuery = 'SELECT "subjectid", "description" FROM "movies"'
curs.execute(sourceQuery)

#获取SQL查询结果
sourceQueryResults = curs.fetchall()

#遍历查询结果
for sourceQueryResult in sourceQueryResults:
  sourceSubjectId = sourceQueryResult[0]
  
  #打印
  print sourceSubjectId
  
  sourceText = sourceQueryResult[1].encode('utf8')
  
  #print sourceText
    
  #进行分词
  algorResults = mmseg.Algorithm(sourceText)
  
  #定义空列表
  tokList = []
  
  #遍历分词结果
  for algorTok in algorResults:
    
    if algorTok.text not in filterList:
      #print algorTok.text.decode('utf8')
      tokList.append(algorTok.text.decode('utf8'))
  
  #排序
  tokList.sort()
  
  #打印测试
  #print tokList
  
  #词频统计
  movieDic = {}
  for tokA in tokList:
    movieDic.update({tokA:tokList.count(tokA)})

  insertQuery = 'update movies set segment = "' + str(movieDic) + '" where subjectid = ' + str(sourceSubjectId)
  
  curs.execute(insertQuery)
  
  conn.commit()

#关闭数据库连接
conn.close()

#字典与字符串转换eval()
